import asyncio
import sys
from collections import Counter

from monkey.common.common_tools import gen_stop_words, text_seg
from monkey.database.motor_base import MotorBase

mongo_db = MotorBase().get_db()
stop_words = gen_stop_words()

async def gen_doc_word_id():
    """
    为单词以及资源文档生成id
    :return:
    """
    cursor = mongo_db.source_docs.find({})
    word_list = []
    doc_id, word_id = 0, 0
    #每一个爬取的内容编号
    async for document in cursor:
        seg_title = text_seg(text=document['title'], stop_words=stop_words)
        doc_id += 1
        cur_item_data = {
            'doc_id': doc_id,
            'seg_title': seg_title,
            "seg_title_counter": Counter(seg_title),
            'title': document['title'],
            'url': document['url']
        }

        await mongo_db.doc_id.update_one({
            'title': cur_item_data['title']},
            {'$set': cur_item_data},
            upsert=True)

        word_list += seg_title
    #为分词的内容编号tf为频率
    for key, value in Counter(word_list).items():
        word_id += 1
        cur_item_data = {
            'word_id': word_id,
            'word': key,
            'tf': value
        }

        await mongo_db.word_id.update_one({
            'word': cur_item_data['word']},
            {'$set': cur_item_data},
            upsert=True)

if __name__ == '__main__':
    asyncio.get_event_loop().run_until_complete(gen_doc_word_id())